package com.guiji.quartz.domain;

import cn.hutool.http.HttpException;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.task.DataSaveUtil;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: wangxiaowen
 * @create: 2021-11-19 12:55
 **/
public class FxNewsGroup {
	private static final Logger log = LoggerFactory.getLogger(FxNewsGroup.class);

	public static void mainMethod() throws Exception {
		String content = null;
		CloseableHttpClient httpClient = null;
		try {
			httpClient = HttpClients.createDefault();
			HttpGet httpGet = new HttpGet("https://fxnewsgroup.com/");
			CloseableHttpResponse resp = httpClient.execute(httpGet);
			if (resp.getStatusLine().getStatusCode() == 200) {
				HttpEntity httpEntity = resp.getEntity();
				content = EntityUtils.toString(httpEntity, "utf8");
			}
			Set<String> resUrl = getUrl(content);
			getContent(resUrl);
		} catch (Exception e) {
			log.error("https://fxnewsgroup.com/ 代理失败1" + e);
			return;
		} finally {
			if (null != httpClient) {
				httpClient.close();
			}
		}


	}

	private static void getContent(Set<String> listContent) throws InterruptedException, IOException {
		for (String url : listContent) {
			Thread.sleep(1000);
			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc = null;
			JXDocument jxDocument = null;
			CloseableHttpClient httpClient = null;
			try {
				httpClient = HttpClients.createDefault();
				HttpGet httpGet = new HttpGet(url);
				CloseableHttpResponse resp = httpClient.execute(httpGet);
				if (resp.getStatusLine().getStatusCode() == 200) {
					HttpEntity httpEntity = resp.getEntity();
					doc = EntityUtils.toString(httpEntity, "utf8");
				}
				if (null == doc || doc.isBlank() || doc.isEmpty()) {
					log.info("请求详情页为空");
					continue;
				}
				jxDocument = new JXDocument(doc);
			} catch (HttpException e) {
				log.info("请求详情页失败" + e);
				continue;
			} finally {
				if (null != httpClient) {
					httpClient.close();
				}
			}
			String title = null;
			try {
				title = (String) jxDocument.sel("//h1/text()").get(0);
			} catch (Exception e) {
				log.error("标题错误-------------" + url + "-------------------标题错误");
				continue;

			}

			String content = null;
			try {
				Document document = Jsoup.parse(doc);
				content = document.getElementsByClass("post-wrap").get(0).toString();
				content = content.replaceAll("</p>.*?\n", "\\$\\$\\$")
					.replaceAll("</ul>.*?\n", "\\$\\$\\$")
					.replaceAll("</li>.*?\n", "\\$\\$\\$")
					.replaceAll("\n","").replaceAll("<div id=.*?</div>", " ")
					.replaceAll("<(.|\n)*?>", " ");
			} catch (Exception e) {
				continue;
			}

			List<Object> newImgs = new ArrayList<>();
			try {
				jxDocument.sel("//article//img/@src").forEach(s -> {
					if (s.toString().startsWith("http")) {
						String temp = s.toString();
						newImgs.add(temp);
					}
				});
			} catch (Exception e) {
				log.error("可能没有图片");
			}

			Date date = new Date();

			if (null != content && content.length() > 50000 || content.replaceAll("\\$|[\\s\\p{Zs}]", "").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			try {
				DataSaveUtil.saveData(title, content, newImgs, url, date, "https://fxnewsgroup.com/", "Fxnewsgroup-Forex");
				log.info(Thread.currentThread().getName() + "-----------------Fxnewsgroup-Forex----------------");
			} catch (Exception e) {
				continue;
			}
		}
	}

	private static Set<String> getUrl(String content) throws Exception {
		JXDocument jxDocument = new JXDocument(content);
		Set<String> res = new HashSet<>();
		String url = null;
		int imgSize = jxDocument.sel("//div[@id='et-recent-posts-3']//li/a/@href").size();
		if (imgSize > 0) {
			String temp = null;
			for (int k = 0; k < imgSize; k++) {
				temp = jxDocument.sel("//div[@id='et-recent-posts-3']//li/a/@href").get(k).toString();
				url = temp.strip();
				res.add(url);
			}
		}
		return res;
	}

}


